import numpy as np

import pandas as pd
import scipy

'''
补充缺失信息
'''
df = pd.read_csv('data/missing_chi.csv')
df.isna().head()
df.isna().mean() # 查看缺失的比例
df.y.value_counts(normalize=True)#统计各个值比例
# 利用缺失值进行填充数据
cat_1 = df.X_1.fillna('NaN').mask(df.X_1.notna()).fillna("NotNaN")
cat_2 = df.X_2.fillna('NaN').mask(df.X_2.notna()).fillna("NotNaN")
# 根据cat1 进行统计y
df_1 = pd.crosstab(cat_1, df.y, margins=True)
df_2 = pd.crosstab(cat_2, df.y, margins=True)
# 根据公式计算特征缺失中正例的理论值
def compute_S(my_df):
    S = []
    for i in range(2):
        for j in range(2):
            E = my_df.iat[i, j]
            F = my_df.iat[i, 2]*my_df.iat[2, j]/my_df.iat[2,2]
            S.append((E-F)**2/F)
    return sum(S)
res1 = compute_S(df_1)
res2 = compute_S(df_2)
# 根据s值判断相关性
from scipy.stats import chi2
scipy.stats.chi2.sf(res1, 1)
scipy.stats.chi2.sf(res2, 1)
from scipy.stats import chi2_contingency
chi2_contingency(pd.crosstab(cat_1, df.y), correction=False)[1]
chi2_contingency(pd.crosstab(cat_2, df.y), correction=False)[1]

#2
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor

df = pd.read_excel('data/color.xlsx')
df_dummies = pd.get_dummies(df.Color)
stack_list = []
'''
创建knn 对象进行训练数据 x,y
对每一个进行最大的
'''
for col in df_dummies.columns:
    clf = KNeighborsRegressor(n_neighbors=6)
    clf.fit(df.iloc[:,:2].values, df_dummies[col].values)
    res = clf.predict([[0.8, -0.2]]).reshape(-1,1)
    stack_list.append(res)
code_res = pd.Series(np.hstack(stack_list).argmax(1))
df_dummies.columns[code_res[0]]

df = pd.read_csv('data/audit.csv')
res_df = df.copy()
# 对年龄 收入 小时进行归一化
df = pd.concat([pd.get_dummies(df[['Marital', 'Gender']]),
    df[['Age','Income','Hours']].apply(
        lambda x:(x-x.min())/(x.max()-x.min())), df.Employment],1)
X_train = df.query('Employment.notna()')
X_test = df.query('Employment.isna()')
df_dummies = pd.get_dummies(X_train.Employment)
'''
针对最后一列分别设置knn 进行对缺失的值进行knn预测生成预测值
'''
for col in df_dummies.columns:
    clf = KNeighborsRegressor(n_neighbors=6)
    clf.fit(X_train.iloc[:,:-1].values, df_dummies[col].values)
    res = clf.predict(X_test.iloc[:,:-1].values).reshape(-1,1)
    stack_list.append(res)
code_res = pd.Series(np.hstack(stack_list).argmax(1))
cat_res = code_res.replace(dict(zip(list(
            range(df_dummies.shape[0])),df_dummies.columns)))
res_df.loc[res_df.Employment.isna(), 'Employment'] = cat_res.values